package com.spider.ask120.webmagic;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.wilddog.client.Wilddog;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验 频道
 * @author Jxw
 *
 */
public class JingyanCatalogBAK  implements PageProcessor{
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 
	 
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
	Wilddog dog = new Wilddog("https://testask.wilddogio.com");

	public void process(Page page) { 
		//进入频道页,获取各病种列表页URL
		if(page.getUrl().regex(jkjyUrl+"$").match()) {
			//保存经验频道
			dog.setValue("jingyan");
			dog.child("jingyan/value").setValue("健康经验");
			//获取到话题页列表URL 
			List<String> huati = page.getHtml().links().regex(listUrl).all(); 
			page.addTargetRequests(huati);
		 
		}else if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$" 
			String catalog = page.getRequest().getExtra("cat") == null ? "" : page.getRequest().getExtra("cat").toString().trim();

			String cataName = page.getHtml().$("#classname","text").get();
			//从URL中获取当前目录名称
			String path = page.getUrl().toString();
			String key = "";
			if (path.indexOf(".html") > 0) {
				key = path.substring(path.indexOf("list/")+5, path.indexOf(".html"));
			} else {
				key = path.substring(path.indexOf("list/")+5);
			} 
			 
			//保存频道、栏目数据
			Wilddog jingyan = dog.child("jingyan/"+catalog+"child/");
			jingyan.child(key+"/value").setValue(cataName);
			 
			
			//判断该页面中如果还有 下一级 栏目， 则继续获取连接
			if(page.getHtml().$(".s_content").match()){
				List<String> huati = page.getHtml().$(".s_content").links().regex(listUrl).all(); 

				Map<String, Object> extras = new HashMap<String, Object>();
				extras.put("cat", catalog+= (catalog.isEmpty()?"":"/") +"child/"+key+"/");
				
				for(String url : huati){ 
					Request request = new Request(url);
					request.setExtras(extras);
					page.addTargetRequest(request);
				}
			} 
			 
		} 
		 

	}
  
	public static void main(String[] args) {
		Spider.create(new JingyanCatalogBAK()).addUrl("http://m.120ask.com/jingyan/list/1kwvmhsksk.html").thread(1).run(); 
		
    }
 
}
